#! /usr/bin/env python3
# coding=utf-8

import re
import os
import time
import random
import asyncio
import aiohttp
import tkinter as tk
from lxml import etree
from selenium import webdriver
from Yanzhao_input_GUI import main_GUI
from selenium.webdriver.chrome.options import Options

# 查询程序
class main_run(main_GUI):
    def __init__(self):
        run_start_time = time.time()
        self.base_url = 'https://yz.chsi.com.cn/zsml/queryAction.do'

        # 打开筛选条件界面
        main_GUI.__init__(self)

        # 程序运行界面
        self.run_gui = tk.Tk()
        self.run_gui.title('招研网爬虫程序提示信息')
        tk.Label(self.run_gui, text='网址：'+self.base_url, anchor=tk.W).pack()
        self.run_gui.update()

        # 参数确认
        self.paras = self.ssdm + self.dwmc + self.mldm + self.yjxkdm + self.zymc + self.xxfs
        if self.yjxkdm == '' or self.yjxkdm == self.paras:
            print('学科类别为必选项，其他至少填写一项')
            tk.Label(self.run_gui, text='学科类别为必选项，其他至少填写一项', fg='red').pack()
            self.run_gui.update()
            time.sleep(3)
            exit()
        print(f'省份:{self.ssdm}\n单位名称:{self.dwmc}\n门类:{self.mldm}\n学科:{self.yjxkdm}\n专业:{self.zymc}\n学习方式:{self.xxfs}')

        # 获得页面链接
        self.t_urls = []
        self.Get_third_urls()
        print('即将获取 {} 个研究生专业目录'.format(len(self.t_urls)))
        tk.Label(self.run_gui, text='即将获取 {} 个研究生专业目录'.format(len(self.t_urls)), anchor=tk.W).pack()
        self.run_gui.update()    

        # 获取页面信息
        self.Get_thrid_asyncio_inf(self.t_urls)

        # 完成时的提示信息
        print(f'资料保存在路径：{os.getcwd()}')
        tk.Label(self.run_gui, text=f'资料保存在：{os.getcwd()}\\{self.paras}考研专业目录详细信息.txt', bg='LawnGreen').pack()
        run_end_time = time.time()
        minute, sec = divmod(run_end_time-run_start_time, 60)
        print('程序共用时 {} 分 {} '.format(minute, int(sec)))
        tk.Label(self.run_gui, text='程序共用时 {} 分 {} '.format(minute, int(sec))).pack()
        self.run_gui.mainloop()

    # 将第三级链接返还给 self.t_urls
    def Get_third_urls(self):
        if (os.path.isfile(self.paras + 'third_url.txt')):
            with open(self.paras + 'third_url.txt', 'r') as f:
                self.t_urls = f.readlines()
        else:
            st_time = time.time()
            self.Thrid_url(self.base_url, self.ssdm, self.dwmc, self.mldm, self.yjxkdm, self.zymc, self.xxfs)
            et_time = time.time()
            print('获取第三级链接用时：{:.3f} s'.format(et_time-st_time))
            tk.Label(self.run_gui, text='获取第三级链接用时：{:.3f} s'.format(et_time-st_time), anchor=tk.W).pack()
            self.run_gui.update()

    # 模拟浏览器，获取第三级链接
    def Thrid_url(self, url, s1='', i1='', s2='', s3='', s4='', s5=''):
        # 先获取第二级链接
        if os.path.isfile(self.paras + 'second_url.txt'):
            with open(self.paras + 'second_url.txt', 'r') as f:
                second_url = f.readlines()
        else:
            chrome_options = Options()
            chrome_options.add_argument('--headless')
            chrome_options.add_argument('--disable-gpu')
            chrome_options.add_argument(
                'user-agent="Mozilla/5.0 (iPod; U; CPU iPhone OS 2_1 like Mac OS X; ja-jp) AppleWebKit/525.18.1 (KHTML, like Gecko) Version/3.1.1 Mobile/5F137 Safari/525.20"')
            print('打开浏览器...', end='')
            tk.Label(self.run_gui, text='打开浏览器...').pack()
            self.run_gui.update()
            driver = webdriver.Chrome(chrome_options=chrome_options)
            driver.get(url)
            # 设置随机时间等待浏览器响应（下同）
            time.sleep(random.randint(3, 5))

            try:
                print('输入条件...', end='')
                tk.Label(self.run_gui, text='输入条件...').pack()
                self.run_gui.update()
                if s1 != '':
                    driver.find_element_by_id('ssdm').click()
                    time.sleep(random.randint(1, 2))
                    driver.find_element_by_xpath('//option[@value="' + s1 + '"]').click()
                    time.sleep(random.randint(1, 2))

                if i1 != '':
                    driver.find_element_by_id('dwmc').send_keys(i1)
                    time.sleep(random.randint(1, 2))

                if s2 != '':
                    driver.find_element_by_id('mldm').click()
                    time.sleep(random.randint(1, 2))
                    driver.find_element_by_xpath('//option[@value="' + s2 + '"]').click()
                    time.sleep(random.randint(1, 2))

                driver.find_element_by_id('yjxkdm').click()
                time.sleep(random.randint(1, 2))
                driver.find_element_by_xpath('//option[@value="' + s3 + '"]').click()
                time.sleep(random.randint(1, 2))

                if s4 != '':
                    driver.find_element_by_id('zymc').click()
                    time.sleep(random.randint(1, 2))
                    driver.find_element_by_xpath('//option[@value="' + s4 + '"]').click()
                    time.sleep(random.randint(1, 2))

                if s5 != '':
                    driver.find_element_by_name('xxfs').click()
                    time.sleep(random.randint(1, 2))
                    driver.find_element_by_xpath('//option[@value="' + s5 + '"]').click()
                    time.sleep(random.randint(1, 2))

                print('查询')
                tk.Label(self.run_gui, text='查询').pack()
                self.run_gui.update()
                driver.find_element_by_name('button').click()
                time.sleep(random.randint(3, 5))

                second_element = driver.find_elements_by_xpath('//form[@id="form3"]/a')
                second_url = []
                for element in second_element:
                    second_url.append(element.get_attribute('href'))
                with open(self.paras + 'second_url.txt', 'w', encoding='utf-8') as f:
                    f.write('\n'.join(second_url))
            except:
                # 浏览器中出错时自动截屏（空图是网络问题，其余为输入参数找不到的错误）
                print(f'浏览器错误')
                driver.save_screenshot('Error.png')
                tk.Label(self.run_gui, text='浏览器错误', fg='red').pack()
                self.run_gui.update()
                time.sleep(5)
                exit()
            finally:
                driver.close()

        # 完成第二级链接获取时的提示
        print('共获得 {} 个学校。'.format(len(second_url)))
        tk.Label(self.run_gui, text='共获得 {} 个学校。'.format(len(second_url))).pack()
        self.run_gui.update()

        # 异步方法打开第二级链接，保持第三级链接至指定文件
        async def third(link):
            session = aiohttp.ClientSession()
            resp = await session.get(link)
            html = await resp.text()
            tree = etree.HTML(html)
            third_url = tree.xpath('//td[@class="ch-table-center"]/a/@href')
            await session.close()
            self.t_urls = self.t_urls + third_url[::2]
            return '\n'.join(third_url[::2])

        async def quest(link):
            result = await third(link)
            with open(self.paras + 'third_url.txt', 'a', encoding='utf-8') as f:
                f.write(result + '\n')

        tasks = [asyncio.ensure_future(quest(s_url)) for s_url in second_url]
        loop = asyncio.get_event_loop()
        loop.run_until_complete(asyncio.wait(tasks))
        return self.t_urls

    # 异步方法获取第三级链接中的信息
    def Get_thrid_asyncio_inf(self, links):
        async def Inf(link, i):
            session = aiohttp.ClientSession()
            # 请求头有没有无所谓
            header = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
            }
            resp = await session.get(link, headers=header)
            time.sleep(random.randint(2, 4))
            if resp.status != 200: print('err')
            html = await resp.text()
            tree = etree.HTML(html)
            inf = tree.xpath('//tbody/tr/td//text()')
            print(i, link)
            inf = '\t'.join(inf)
            inf = re.sub('[\r\n]', '', inf)
            inf = re.sub('见招生简章', '', inf)
            inf = re.sub('：[\t]+', '：', inf)
            inf = re.sub('[\t ]+', '\n', inf)
            await session.close()
            return inf

        # 保存信息至指定文件夹
        async def waite(link, i):
            result = await Inf(link, i)
            f.write(str(i) + ':' + link + '\n' + result + '\n')

        head = 'https://yz.chsi.com.cn'
        links = [re.sub('[\r\n]', '', head + l) for l in links]
        f = open(self.paras + '考研专业目录详细信息.txt', 'a', encoding='utf-8')
        task = [asyncio.ensure_future(waite(link, key)) for key, link in enumerate(links)]
        loop = asyncio.get_event_loop()
        loop.run_until_complete(asyncio.wait(task))
        f.close()

if __name__ == '__main__':
    main_run()